# -*- coding: utf-8 -*-
import re

import scrapy


class ScrapyqSpider(scrapy.Spider):
    name = 'scrapyq'
    allowed_domains = ['osgeo.cn']
    start_urls = ['https://www.osgeo.cn/scrapy/index.html']
    url_head = "https://www.osgeo.cn/scrapy/"

    def parse(self, response):
        ret = response.body.decode()

        # 获取css文件路径
        css_list = re.findall(r'href="(.*?)"\stype="text/css"\s/>', ret)
        # 获取js文件路径
        js_list = re.findall(r'src="(.*?\.js)"', ret)
        # 获取html文件路径
        html_list = re.findall(r'href="(.*?\.html)"', ret)

        yield {
            "type": "html",
            "data": ret,
            "path": response.url
        }

        for h in html_list:
            yield scrapy.Request(url=self.url_head + h, callback=self.html_parse)

        for j in js_list:
            yield scrapy.Request(url=self.url_head + j, callback=self.js_parse)

        for c in css_list:
            yield scrapy.Request(url=self.url_head + c, callback=self.css_parse)

    def html_parse(self, response):
        ret = response.body.decode()

        css_list = re.findall(r'href="(.*?)"\stype="text/css"\s/>', ret)

        js_list = re.findall(r'src="(.*?\.js)"', ret)

        html_list = re.findall(r'href="(.*?\.html)"', ret)

        yield {
            "type": "html",
            "data": ret,
            "path": response.url
        }

        for h in html_list:
            yield scrapy.Request(url=self.url_head + h, callback=self.html_parse)

        for j in js_list:
            yield scrapy.Request(url=self.url_head + j, callback=self.js_parse)

        for c in css_list:
            yield scrapy.Request(url=self.url_head + c, callback=self.css_parse)

    def js_parse(self, response):

        yield {
            "type": "js",
            "data": response.body.decode(),
            "path": response.url
        }

    def css_parse(self, response):

        yield {
            "type": "css",
            "data": response.body.decode(),
            "path": response.url
        }
